#!/usr/bin/env python
# coding=utf-8

import os, sys, re, traceback, logging
import urllib
from datetime import *
import time
import datetime
import getopt
import urllib2
from Route import *

#获取国内及国外线路url入口
vacation_page = re.compile(r'''<div\s*class=.detail_title.[\S\s]+?</td>''', re.X|re.S)
vacation_page_city = re.compile(r'''<a\s*href="([\S\s]+?)"[\S\s]+?>([\S\s]+?)<''', re.X|re.S)
vacation_pagenum = re.compile(r'''<div\s*style=.line-height[\S\s]+?总共([0-9]+?)页''', re.X|re.S)

route_url_re = re.compile(r'''<div\s*class=.detail_001_b_002.\s*>\s*<a\s*href="([\S\s]+?)"[\S\s]+?(<li\s*class="black12_b">[\S\s]+?</table>)''', re.X|re.S)
route_dates_re = re.compile(r'''<li\s*class="black12_b">出发日期[\S\s]+?</table>''', re.X|re.S)
route_dates2_re = re.compile(r'''<td\s*width=.85.\s*align=.left.>([\S\s]+?)</td>''', re.X|re.S)

re_h=re.compile('<[\S\s]*>') #取出信息中夹杂的html标签
http_url = "http://www.cits.com.cn"  #国旅在线
route_telphone = "400-600-8888"

def do_crawl():
    try:
        server_url_data = [
                ('国内','跟团游', 'http://www.cits.com.cn/citsonlineWeb/switchdo.do?prefix=/domestic&page=/domesticRouteQuery.do', ),
                ('出境','跟团游', 'http://www.cits.com.cn/citsonlineWeb/switchdo.do?prefix=/outbound&page=/indexOutboundRouteQuery.do', ),
        ]
        #
        for country,routetype,server_url in server_url_data:
            html = urllib2.urlopen(server_url).read()
            v_page = vacation_page.findall(html)
            if v_page:
                v_page_c = vacation_page_city.findall(v_page[0])
                if v_page_c:
                    for p_url, dep_city in v_page_c:
                        p_urls = http_url+p_url
                        try:
                            html = urllib2.urlopen(p_urls).read()
                            page_n = vacation_pagenum.findall(html)
                            if page_n:
                                print page_n[0]+"========"
                                for i in range(int(page_n[0])):
                                    p_urls = http_url+p_url+"&nowPage="+str(i+1)
                                    #print p_urls
                                    try:
                                        p_html = urllib2.urlopen(p_urls).read()
                                        route_urls = route_url_re.findall(p_html)
                                        if route_urls:
                                            for route_url, r_dates in route_urls:
                                                print "rotue_url:"+route_url
                                                route_dates =""
                                                if r_dates:
                                                    r_dates2 = route_dates2_re.findall(r_dates)
                                                    if r_dates2:
                                                        for i in r_dates2:
                                                            if i.replace("&nbsp;",""):
                                                                route_dates = route_dates+i+";"
                                                        #print "route_dates:"+route_dates
                                                getRoutes(route_url,route_dates,dep_city,routetype)
                                    except:
                                        print traceback.format_exc(sys.exc_info())
                        except:
                            print traceback.format_exc(sys.exc_info())
    except:
        print traceback.format_exc(sys.exc_info())


route_title_re = re.compile(r'''<div\s*class="detail_003_title"\s*>([\S\s]+?)</div>''', re.X|re.S)
route_price_re = re.compile(r'''网上支付价：([0-9].*)元起\s*</td>''', re.X|re.S)
route_day_re = re.compile(r'''行程天数：([0-9]+?)天''', re.X|re.S)
route_detail_re = re.compile(r'''线路特色：</span><br/>\s*([\S\s]+?)\s*</li>''', re.X|re.S)

route_planes_re = re.compile(r'''<div\s*class=.detail_004_002_left2.\s*>[\S\s]+?<li>([\S\s]+?)\s*</li>''', re.X|re.S)
route_planes2_re = re.compile(r'''第[\S\s]+?天：([\S\s]+?)<[\S\s]+?<br\s*/>([\S\s]+?)<br\s*/>''', re.X|re.S)
route_payown_re = re.compile(r'''<td\s*id=.detail_005_body_02.[\S\s]+?>([\S\s]+?)\s*</td>''', re.X|re.S)
route_trip_all_re = re.compile(r'''<td\s*id=.detail_005_body_01.[\S\s]+?>([\S\s]+?)\s*</td>''', re.X|re.S)
route_sights_re = re.compile(r'''<div\s*class=.detail_004_002_a[\S\s]+?天([\S\s]+?)经停城市:\s*</div>''', re.X|re.S)


def getRoutes(route_url,route_dates,depcity,routetype):
    print "route start"
    #route_url = "http://www.cits.com.cn//outbound/route/04001016855.htm"
    #此处开始获取线路信息
    try:
        html = urllib2.urlopen(route_url).read()
        #html = getUTF(html)
        route_title = route_title_re.findall(html)
        if route_title:
            route_title = route_title[0]
            print route_title
        #
        if route_title.find("自由")>-1:
            routetype = "自由行"
        #
        route_price = route_price_re.findall(html)
        if route_price:
            route_price = route_price[0]
            #print route_price
        #
        route_days = route_day_re.findall(html)
        if route_days:
            route_days = route_days[0]
        else:
            route_days = 1
        #print route_days
        #
        route_detail = route_detail_re.findall(html)
        if route_detail:
            route_detail = route_detail[0].strip()
            #print route_detail
        #
        route_sights =""
        route_sight = route_sights_re.findall(html)
        if route_sight:
            for i in route_sight:
                route_sightss = i.replace("&nbsp;","").replace("-",",").replace("—",",").replace("：","").replace(depcity,"").strip()
                sightss = route_sightss.split(",")
                for k in sightss:
                    if route_sights.find(k)<0:
                        route_sights = route_sights + k+","
        #
        route_plane = ""
        place_list = route_planes_re.findall(html)
        if place_list:
            plane_num =1
            for detail in place_list:
                route_plane = route_plane + "#第" + str(plane_num) + "天#"
                route_plane = route_plane + detail.strip()
                plane_num = plane_num+1
        else:
            place_list = route_planes2_re.findall(html)
            if place_list:
                plane_num =1
                for sign,detail in place_list:
                    route_sightss = sign.replace("&mdash;",",").replace("-",",").replace("－",",").replace("－",",")
                    sightss = route_sightss.split(",")
                    for k in sightss:
                        if route_sights.find(k)<0:
                            route_sights = route_sights + k+","
                    route_plane = route_plane + "#第" + str(plane_num) + "天#"
                    route_plane = route_plane + strip_tags(detail).strip()
                    plane_num = plane_num+1
        #print route_plane
        #
        #print route_sights
        #
        route_hotel = ""
        route_meal = ""
        route_traffic01 = ""
        route_tickets = ""
        route_payown = ""
        route_trip = route_trip_all_re.findall(html)
        if route_trip:
            if route_trip[0].find("<br>")>-1:
                route_trip = route_trip[0].split("<br>")
            else:    
                route_trip = route_trip[0].split("；")
            for route_trips in route_trip:
                if route_trips.find("住宿")>-1:
                    if route_trips.find("：")>-1:
                        route_trips = (route_trips.split("："))[1]
                    route_hotel = re_h.sub('',route_trips)
                if route_trips.find("用餐")>-1:
                    if route_trips.find("：")>-1:
                        route_trips = (route_trips.split("："))[1]
                    route_meal = re_h.sub('',route_trips)
                if route_trips.find("用车")>-1:
                    if route_trips.find("：")>-1:
                        route_trips = (route_trips.split("："))[1]
                    route_traffic01 = re_h.sub('',route_trips)                 
                if route_trips.find("门票")>-1:
                    if route_trips.find("：")>-1:
                        route_trips = (route_trips.split("："))[1]
                    route_tickets = re_h.sub('',route_trips)
        #print route_hotel
        #print route_meal
        #print route_traffic01
        #print route_tickets
        #            
        route_payown = route_payown_re.findall(html)
        if route_payown:
            route_payown =route_payown[0]
        #print route_payown
        #
        route_go_back = getTrafficType(route_title)
        #print route_go_back
        #入库
        route = Route()
        route.site = '26'
        route.supplier_url = "http://www.cits.com.cn/"
        route.supplier = "国旅在线"
        route.title = route_title
        route.rout_url = route_url
        route.outcity = depcity
        route.sights = route_sights
        route.telphone = route_telphone
        route.detail = route_detail
        route.dest = route_sights
        route.guide = ""
        route.type = routetype
        route.dates = route_dates
        route.price = int(route_price)
        route.go_t = route.back_t = route_go_back
        route.plane = route_plane
        route.days = route_days
    
        route.hotel = route_hotel
        route.meal = route_meal
        route.traffic01 = route_traffic01
        route.tickets = route_tickets
        route.payown = route_payown
        
        getschedules = getSchedules(html)
        if getschedules=='no':
            print "==no schedule=="
        else:
            route.schedules = getschedules
            route.updRoute_Schedules()
        route.updRoute()
        print "++++++"
    except:
        print traceback.format_exc(sys.exc_info())
    #print "++++++"
    
Schedules_re = re.compile(r'''<div\s*class=.detail_004_002_left2.\s*>[\S\s]+?(<li>[\S\s]+?</li>)\s*</ul>''', re.X|re.S)
Schedules_brief_re = re.compile(r'''<li[\S\s]+?</li>''', re.X|re.S)
Schedules_place_re = re.compile(r'''''', re.X|re.S)
Schedules_eat_tra_hot_re = re.compile(r'''(<span>[\S\s]+)''', re.X|re.S)


def getSchedules(html):
     
     schedules_list = Schedules_re.findall(html)
     if schedules_list:
         schedules = range(0,len(schedules_list))
         n = 0
         s_sights = route_sights_re.findall(html)
         for plane_info in schedules_list:
            n+=1
            schedule = Schedule()
            s_brief = Schedules_brief_re.findall(plane_info)
            if s_brief:
                schedule.s_brief = strip_tags(s_brief[0]).strip()
            #print "brief:"+strip_tags(s_brief[0]).strip()
            
            if len(s_sights)>n-1:
                s_sight = s_sights[n-1]
                s_sight= s_sight.replace("&nbsp;","").replace("-",",").replace("—",",").replace("：","").strip()
                if s_sight.find(",")>-1:
                    s_sight = s_sight.split(",")[1]
                schedule.s_place = s_sight
                #print "s:"+s_sight
            if len(s_brief)>2:
                schedule.s_hotel = strip_tags(s_brief[2]).strip().replace("住宿：","")
            #print strip_tags(s_brief[2]).strip().replace("住宿：","")
            schedule.s_traffic = ""
            
            if len(s_brief)>1:
                s_bld = strip_tags(s_brief[1]).strip().replace("用餐：","")
                #print s_bld+"====================="
                if s_bld.find("不")>-1:
                    s_bld = s_bld.split("不")[0]
                #print schedule_hbld
                schedule.s_brf="1" if s_bld.find('早')>-1 else "0"
                schedule.s_lunch="1" if s_bld.find('中')>-1 else "0"
                schedule.s_dinner="1" if s_bld.find('晚')>-1 else "0"
            
            schedule.s_num = n
            #print "serialnumber:"+str(n)
            schedules[n-1]=schedule
         return schedules
         
     return "no"
 
#自由行
vacation_pagenum_z = re.compile(r'''总共([0-9]+?)页''', re.X|re.S)
route_url_re_z = re.compile(r'''<div\s*class=.zyx_detail_001_b_001[\S\s]+?>\s*<a\s*href=".([\S\s]+?)"[\S\s]+?title="([\S\s]+?)">([\S\s]+?)</a>\s*</div>''', re.X|re.S)

def do_crawl_z():
    try:
        server_url_data = [
                ('国内、国外','自由行', 'http://www.cits.com.cn/citsonlineWeb/switchdo.do?prefix=/selftrip&page=/productQueryB2CAction.do?actionType=Init', ),
        ]
        #
        for country,routetype,server_url in server_url_data:
            try:
                html = urllib2.urlopen(server_url).read()
                page_n = vacation_pagenum_z.findall(html)
                if page_n:
                    print page_n[0]+"========"
                    for i in range(int(page_n[0])):
                        p_urls = server_url+"&orderby=%20&currentPage="+str(i+1)
                        #print p_urls
                        try:
                            p_html = urllib2.urlopen(p_urls).read()
                            route_urls = route_url_re_z.findall(p_html)
                            if route_urls:
                                for route_url, r_title, r_depcity in route_urls:
                                    route_url =  "http://www.cits.com.cn/citsonlineWeb"+route_url
                                    print "rotue_url:"+route_url
                                    print "r_title:"+r_title
                                    if r_depcity.find("北京出发")>-1:
                                        dep_city = "北京"
                                    if r_depcity.find("上海出发")>-1:
                                        dep_city = "上海"
                                    if r_depcity.find("广州出发")>-1:
                                        dep_city = "广州"
                                    if r_depcity.find("沈阳出发")>-1:
                                        dep_city = "沈阳"
                                    print "dep_city"+dep_city
                                    getRoutes_z(route_url,r_title,dep_city,routetype)
                        except:
                            print traceback.format_exc(sys.exc_info())
            except:
                print traceback.format_exc(sys.exc_info())
    except:
        print traceback.format_exc(sys.exc_info())

route_price_re_z = re.compile(r'''成人价：([\S\s]+?)\s*</td>''', re.X|re.S)
route_day_re_z = re.compile(r'''<div\s*class=.zyx_detail_004_002_a[\S\s]+?天\s*([\S\s]+?)\s*</div>''', re.X|re.S)
route_detail_re_z = re.compile(r'''线路特色：([\S\s]+?)</ul>''', re.X|re.S)
route_sights_re_z = re.compile(r'''<div\s*class=.zyx_detail_004_002_a[\S\s]+?天\s*([\S\s]+?)\s*</div>''', re.X|re.S)
route_planes_re_z = re.compile(r'''<td\s*valign=.top.>[\S\s]+?<ul>\s*<li>([\S\s]+?)</li>''', re.X|re.S)

def getRoutes_z(route_url,r_title,depcity,routetype):
    print "route start"
    #此处开始获取线路信息
    try:
        html = urllib2.urlopen(route_url).read()
        #html = getUTF(html)
        route_title = r_title
        #
        route_price = route_price_re_z.findall(html)
        if route_price:
            route_price = strip_tags(route_price[0]).strip().replace(".00","").replace("起","")
            #print route_price
        #
        route_detail = route_detail_re_z.findall(html)
        if route_detail:
            route_detail = re.compile('\s').sub('',strip_tags(route_detail[0]).strip())
            #print route_detail
        #
        route_days = route_day_re_z.findall(html)
        if route_days:
            route_days = len(route_days)
        else:
            route_days = 1
        #print route_days
        #
        route_sights =""
        route_sight = route_sights_re_z.findall(html)
        if route_sight:
            for i in route_sight:
                route_sightss = i.replace("&nbsp;","").replace(" ","").replace("-",",").strip()
                sightss = route_sightss.split(",")
                for k in sightss:
                    if route_sights.find(k)<0:
                        route_sights = route_sights + k+","
        #print route_sights
        #
        route_plane = ""
        place_list = route_planes_re_z.findall(html)
        if place_list:
            plane_num =1
            for detail in place_list:
                route_plane = route_plane + "#第" + str(plane_num) + "天#"
                route_plane = route_plane + detail.strip()
                plane_num = plane_num+1
        #print route_plane
        #
        #
        route_hotel = ""
        route_meal = ""
        route_traffic01 = ""
        route_tickets = ""
        route_payown = ""
        #
        route_go_back = getTrafficType(route_title)
        #print route_go_back
        #入库
        route = Route()
        route.site = '26'
        route.supplier_url = "http://www.cits.com.cn/"
        route.supplier = "国旅在线"
        route.title = route_title
        route.rout_url = route_url
        route.outcity = depcity
        route.sights = route_sights
        route.telphone = route_telphone
        route.detail = route_detail
        route.dest = route_sights
        route.guide = ""
        route.type = routetype
        route.dates = getdates()
        route.price = int(route_price)
        route.go_t = route.back_t = route_go_back
        route.plane = route_plane
        route.days = route_days
    
        route.hotel = route_hotel
        route.meal = route_meal
        route.traffic01 = route_traffic01
        route.tickets = route_tickets
        route.payown = route_payown
        
        getschedules = getSchedules_z(html)
        if getschedules=='no':
            print "==no schedule=="
        else:
            route.schedules = getschedules
            route.updRoute_Schedules()
        route.updRoute()
        print "++++++"
    except:
        print traceback.format_exc(sys.exc_info())
    #print "++++++"
    
Schedules_re_z = re.compile(r'''<div\s*class=.zyx_detail_004_002_a[\S\s]+?天\s*([\S\s]+?)\s*</div>''', re.X|re.S)
Schedules_brief_re_z = re.compile(r'''<td\s*valign=.top.>[\S\s]+?<ul>\s*<li>([\S\s]+?)</li>''', re.X|re.S)

def getSchedules_z(html):
     schedules_list = Schedules_re_z.findall(html)
     if schedules_list:
         schedules = range(0,len(schedules_list))
         n = 0
         s_sights = route_sights_re_z.findall(html)
         for i in range(len(schedules_list)):
            n+=1
            schedule = Schedule()
            s_brief = Schedules_brief_re_z.findall(html)
            if s_brief:
                schedule.s_brief = strip_tags(s_brief[i]).strip()
            #print "brief:"+strip_tags(s_brief[i]).strip()
            
            if len(s_sights)>n-1:
                s_sight = s_sights[i].replace("&nbsp;","").replace(" ","").replace("-",",").strip()
                schedule.s_place = s_sight
                #print "s:"+s_sight
            
            schedule.s_hotel = ""
            schedule.s_traffic = ""
            s_briefs =  s_brief[i]
            if len(s_brief)>1:
                schedule.s_brf="1" if s_briefs.find('早')>-1 else "0"
                schedule.s_lunch="1" if s_briefs.find('中')>-1 else "0"
                schedule.s_dinner="1" if s_briefs.find('晚')>-1 else "0"
            
            schedule.s_num = n
            #print "serialnumber:"+str(n)
            schedules[n-1]=schedule
         return schedules
     return "no" 

def getTrafficType(t):
    t1='-'
    if t.find('大巴')>-1:
        t1='汽车'
    if t.find('汽')>-1:
        t1='汽车'
    if t.find('动')>-1:
        t1='动车'
    if t.find('卧')>-1:
        t1='火车'
    if t.find('火车')>-1:
        t1='火车'
    if t.find('船')>-1:
        t1='邮轮'
    if t.find('邮')>-1:
        t1='邮轮'
    if t.find('飞')>-1:
        t1='飞机'
    if t.find('机')>-1:
        t1='飞机'
    return t1

def getdates():
    #天天发团是 使用该方法
    #from datetime import *  #将此行放在import datetime上边
    #days=i+1  days=?  就是距当前日期的天数。
    route_dates = ""
    date1 = date.today()
    for i in range(10):
        d=datetime.timedelta(days=i+1)
        route_dates = str(route_dates)+str(date1+d)+ ";"
    #print route_dates
    return route_dates

#去掉文本标签
def strip_tags(html):
    from HTMLParser import HTMLParser
    html=html.strip()
    html=html.strip("\n")
    result=[]
    parse=HTMLParser()
    parse.handle_data=result.append
    parse.feed(html)
    parse.close()
    return "".join(result)


if __name__ == '__main__':
    script_path = os.path.dirname(os.path.realpath(__file__))
    os.chdir(script_path)
    do_debug = False

    try:
        opts, args = getopt.gnu_getopt(sys.argv[1:], 'hdno:p:', ['help', 'debug', 'no-cache', 'only-cache', 'provider'])
    except getopt.GetoptError:
        usage()
        sys.exit()

    for opt, arg in opts:
        if opt in ('-h', '--help'):
            usage()
            sys.exit()
        elif opt in ('-d', '--debug'):
            do_debug = True
        elif opt in ('-n', '--no-cache'):
            use_cache = False
        elif opt in ('-o', '--only-cache'):
            if arg.lower() in ('no', 'n', '0'):
                only_cache = False
            else:
                only_cache = True
        elif opt in ('-p', '--provider'):
            pass

    if do_debug:
        import pdb
        pdb.set_trace()
    do_crawl()
    do_crawl_z()